##############
# (*) Kaleidoscope
# By: Allan Spale
# May 2008

from bsddb import db
import sys
import struct
import time
import cPickle as pickle
import cStringIO as StringIO
import uuid

import mx.DateTime

from Kaleidoscope.KSEventManager import KSEventManager
from Kaleidoscope.KSUtility import KSUtility

class KSColumnError(Exception):
    """
    Exception class for KColumn

    """
    B_UNKNOWN_EXCEPTION = True
    B_CLASS_EXCEPTION = False

    def __init__(self, u_message):
        self.__u_message = u_message

    def __str__(self):
        return repr(self.__u_message)

    @staticmethod
    def print_and_raise_error(u_message, b_unknown_exception,
                                ksem_event_manager, i_event_id):
        """
        Convenience method for printing and raising an error for this
        particular Exception subclass.

        """
        # write initial error
        u_message = u'KSColumnError.py: ' + u_message
        sys.stderr.write(u_message + u'\n')

        if (isinstance(ksem_event_manager, KSEventManager)):
            send_event_message = ksem_event_manager.send_event_message
        else:
            send_event_message = lambda x, y: None

        if (b_unknown_exception):
            send_event_message(i_event_id, [KSColumnError(u_message)])
            raise Exception(u_message)
        else:
            send_event_message(i_event_id, [KSColumnError(u_message)])
            raise KSColumnError(u_message)

    @staticmethod
    def print_and_raise_error_no_event(u_message, b_unknown_exception):
        """
        Convenience method for printing and raising an error for this
        particular Exception subclass.

        """
        #sys.stderr.write(sys.exc_info())  # write initial error
        u_message = u'KSColumnError.py: ' + u_message

        if (b_unknown_exception):
            raise Exception(u_message)
        else:
            raise KSColumnError(u_message)


class KSColumn(object):
    """
    This is the base class for all databases. Potential systems for which
    to include support include BerkeleyDB, Metaweb Freebase and
    Google Spreadsheets.

    This module will be a tasklet that will be fronted by a message queue
    that will be used to process trigger commands so that there might be some
    increase in parallelism.

    For all functions that are to be implemented in a subclass, the function
    calls need to check the event channel callbacks to see if any data must
    be sent on channels AT THE START (i.e. before the function executes) or
    AFTER the function finished executing. The format should be the following:

    def tableFunction(self):
        triggerBeforeFunctionEvents(iBefore_tableFunctionEventID)
        # ... tableFunction code ...
        triggerAfterFunctionEvents(iAfter_tableFunctionEventID)
        return whatever

    """
    # Properties

    __now = now = mx.DateTime.now

    MIN_REMAINING_IDS = 60 * 60 * 3 * KSUtility.MAX_IDS_PER_SECOND

    # REDO ALL CONSTANT PROPERTIES TO BE NORMAL/WRITABLE OUTSIDE OF CLASS...
    # FIX THIS LATER
    B_PACKED_DATA = True
    B_REGULAR_DATA = False
    NULL_VALUE = -sys.maxint - 1

    # Key Types
    U_KEY_TYPE_INT = u'int'
    U_KEY_TYPE_LONG = u'long'
    U_KEY_TYPE_UNICODE = u'unicode'

    # Comparison flags
    I_LESS_THAN = 0
    I_LESS_THAN_OR_EQUAL = 1
    I_GREATER_THAN = 2
    I_GREATER_THAN_OR_EQUAL = 3
    I_EQUAL = 4
    I_NOT_EQUAL = 5
    I_SKIP_COMPARISON = 6

    I_UNDEFINED = 0

    B_PACK_KEY = True
    B_PICKLE_KEY = False

    # Use to do a uniqueness check on each value
    B_UNIQUE_VALUES = True
    B_DUPLICATE_VALUES = False

    """
    @property
    def event_manager(self):
        return self.__event_manager
    """

    @property
    def u_database_name(self):
        return self.__u_database_name

    @property
    def u_key_type(self):
        return self.__u_key_type

    @property
    def b_unique_values(self):
        return self.__b_unqiue_values

    @property
    def mx_start_datetime(self):
        return self.__mx_start_datetime

    @property
    def mx_last_datetime(self):
        return self.__mx_last_datetime

    @property
    def uuid(self):
        return self.__uuid


    def __init__(self, u_database_name, mx_start_datetime, u_key_type,
                    b_pack_key, b_unqiue_values):

        # Check whether values for each key must be unique (only appear once
        # in the table). To prevent more complicated code, simply assign a
        # check for uniqueness function that will either execute or not
        # based on whether this constraint was specified.
        self.__b_unqiue_values = False
        self.value_exists_check = None
        self.set_unique_values_flag(b_unqiue_values)
        self.__i_ids_this_second = 0

        u_general_error = u'The database core structure could not be created.'

        #print mx_start_datetime.gmticks()
        self.__time_of_last_id = mx_start_datetime

        # Session-unique ID
        self.__uuid = uuid.uuid4()

        # Function comparison table
        self.__function_comparison = [0, 1, 2, 3, 4, 5, 6]
        self.__function_comparison[KSColumn.I_LESS_THAN] = \
            lambda x, y: (x<y)
        self.__function_comparison[KSColumn.I_LESS_THAN_OR_EQUAL] = \
            lambda x, y: (x<=y)
        self.__function_comparison[KSColumn.I_GREATER_THAN] = \
            lambda x, y: (x>y)
        self.__function_comparison[KSColumn.I_GREATER_THAN_OR_EQUAL] = \
            lambda x, y: (x>=y)
        self.__function_comparison[KSColumn.I_EQUAL] = \
            lambda x, y: (x==y)
        self.__function_comparison[KSColumn.I_NOT_EQUAL] = \
            lambda x, y: (x!=y)
        self.__function_comparison[KSColumn.I_SKIP_COMPARISON] = \
            lambda x, y: True

        # Function assignments based on whether keys should be packed
        # or pickled.
        if (isinstance(b_pack_key, bool)):
            if (b_pack_key):
                self.marshal_key = self.pack_key
                self.unmarshal_key = self.unpack_key
            else:
                self.marshal_key = self.pickle_data
                self.unmarshal_key = self.custom_unpickle_data
        else:
            KSColumnError.print_and_raise_error(u'The pack_key flag '
                + u'must be of type boolean.', KSColumnError.B_CLASS_EXCEPTION)


        try:
            if (isinstance(u_database_name, unicode)):
                self.__u_database_name = u_database_name
            else:
                KSColumnError.print_and_raise_error(u'A valid unicode '
                    + u'is required for naming the column.',
                    KSColumnError.B_CLASS_EXCEPTION)

            self.__type_mx_datetime = type(mx.DateTime.Date(2000, 1, 1))

            if (isinstance(mx_start_datetime, self.__type_mx_datetime)):
                self.__mx_start_datetime = mx_start_datetime
                self.__mx_last_datetime = mx_start_datetime + \
                    mx.DateTime.DateTimeDeltaFromSeconds(
                        (sys.maxint / KSUtility.MAX_IDS_PER_SECOND) \
                        - KSColumn.MIN_REMAINING_IDS)
            else:
                KSColumnError.print_and_raise_error(u'A starting date and '
                    + u'time must be provided to each column in order '
                    + u'to generate IDs.',
                    KSColumnError.B_CLASS_EXCEPTION)

            if (isinstance(u_key_type, unicode)):
                if (u_key_type == KSColumn.U_KEY_TYPE_INT or
                    u_key_type == KSColumn.U_KEY_TYPE_UNICODE or
                    u_key_type == KSColumn.U_KEY_TYPE_LONG):
                    self.__u_key_type = u_key_type
                else:
                    KSColumnError.print_and_raise_error(u'The type of the '
                        + u'key must be specified as int, long, or unicode.',
                        KSColumnError.B_CLASS_EXCEPTION)
            else:
                KSColumnError.print_and_raise_error(u'The type of the key '
                    + u'must be specified as a unicode string.',
                    KSColumnError.B_CLASS_EXCEPTION)


        except db.DBError:
            KSColumnError.print_and_raise_error(
                u_general_error, KSColumnError.B_UNKNOWN_EXCEPTION)


    def __del__(self):
        self.close()


    def set_unique_values_flag(self, b_flag):
        if (isinstance(b_flag, bool)):
            if (b_flag):
                self.__b_unqiue_values = True
                self.value_exists_check = lambda x, y: self.has_value(x, y)
            else:
                self.__b_unqiue_values = False
                self.value_exists_check = lambda x, y: False
        else:
            KSColumnError.print_and_raise_error(u'The b_unqiue_values flag '
                + u'must be of type boolean.', KSColumnError.B_CLASS_EXCEPTION)


    def open(self, i_flags, i_permissions, kstx_parent_transaction):
        """
        Open the file containing the column.

        """
        raise NotImplementedError, \
            u'open must be implemented in a subclass.'


    def close(self):
        """
        Close the file containing the column.

        """
        raise NotImplementedError, \
            u'close must be implemented in a subclass.'


    def get_data_from_key(self, obj_key, kstx_parent_transaction,
                              b_return_packed_data):
        """
        Return the data object based on an integer or string key. Integers
        will be packed according to big endian notation.

        """
        raise NotImplementedError, \
            u'get_data_from_key must be implemented in a subclass.'


    def get_keys_from_data_value(self, obj_target_data,
                                 kstx_parent_transaction, b_return_packed_data):
        """
        Go through the data and find the list of keys from the given values.

        """
        raise NotImplementedError, \
            u'get_keys_from_data_value must be implemented in a subclass.'


    def get_keys_from_data_object_range(self, obj_start_data, obj_end_data,
            i_low_comparison_operator, i_high_comparison_operator,
            kstx_parent_transaction, b_return_packed_data):
        """
        Go through the data and find the list of keys from the given values
        in the range of data items. It is assumed that the caller will use
        a type that would support a range of data. Range is
        obj_start_data <= data_item <= obj_end_data.

        """
        raise NotImplementedError, \
            u'get_keys_from_data_object_range must be implemented in ' \
                + u'a subclass.'


    def get_data_objects_from_key_range(self, obj_start_key, obj_end_key,
            i_low_comparison_operator, i_high_comparison_operator,
            kstx_parent_transaction, b_return_packed_data):
        """
        Go through the keys and find the list of data objects from the given
        values in the range of keys. It is assumed that the caller will use
        a type that would support a range of data. Range is
        obj_start_key <= key <= obj_end_key.

        """
        raise NotImplementedError, \
            u'get_data_objects_from_key_range must be implemented in ' \
                + u'a subclass.'


    def has_key(self, obj_key, kstx_parent_transaction):
        """
        Return a boolean value that indicates whether or not the key exists
        in the database.

        """
        raise NotImplementedError, \
            u'has_key must be implemented in a subclass.'


    def has_value(self, obj_value, kstx_parent_transaction):
        """
        Return a boolean value that indicates whether or not the value exists
        at least once in the database.

        """
        raise NotImplementedError, \
            u'has_value must be implemented in a subclass.'


    def insert_item_with_key_value_pair(self, obj_key, obj_data,
                                            kstx_parent_transaction):
        """
        Insert the key/data pair.

        """
        raise NotImplementedError, u'insert_item_with_key_value_pair ' \
            + u'must be implemented in a subclass.'


    def insert_value_and_generate_key(self, obj_data, kstx_parent_transaction):
        """
        Insert data with time_id as a key. Key will be stored in big endian
        notation.

        """
        raise NotImplementedError, u'insert_value_and_generate_key ' \
            + u'must be implemented in a subclass.'


    def insert_items_with_key_and_value_list(self, obj_key, list_values,
                                                 kstx_parent_transaction):
        """
        Insert data with time_id as a key. Key will be stored in big endian
        notation.

        """
        raise NotImplementedError, u'insert_items_with_key_and_value_list ' \
            + u'must be implemented in a subclass.'


    def change_key(self, obj_old_key, obj_new_key, kstx_parent_transaction):
        """
        Get the data from the old key, and store it as a new key/value pair

        """
        raise NotImplementedError, u'change_key ' \
            + u'must be implemented in a subclass.'


    def change_all_data_values(self, obj_old_data_value, obj_new_data_value,
                                   kstx_parent_transaction):
        """
        Change all data values to another value. Slow way: scan the database
        and swap out the old values with the new values.

        """
        raise NotImplementedError, u'change_data_value ' \
            + u'must be implemented in a subclass.'


    def change_key_value_pair(self, obj_key, obj_data, kstx_parent_transaction):
        """
        Replace the old value of a key with a new value.

        """
        raise NotImplementedError, u'change_key_value_pair ' \
            + u'must be implemented in a subclass.'


    def delete_item_by_key(self, obj_key, kstx_parent_transaction):
        """
        Delete the item represented by the given key.

        """
        raise NotImplementedError, u'delete_item_by_key ' \
            + u'must be implemented in a subclass.'


    def delete_items_by_value_list(self, list_values, kstx_parent_transaction):
        """
        Change all data values to another value. Slow way: scan the database
        and swap out the old values with the new values.

        """
        raise NotImplementedError, u'delete_items_by_value_list ' \
            + u'must be implemented in a subclass.'


    def get_all(self, kstx_parent_transaction, b_return_packed_data):
        """
        Retrieve entire database of items, both keys and values.

        """
        raise NotImplementedError, u'get_all ' \
            + u'must be implemented in a subclass.'


    def values(self, kstx_parent_transaction, b_return_packed_data):
        """
        Retrieve all values from the database of items.

        """
        raise NotImplementedError, u'values ' \
            + u'must be implemented in a subclass.'


    def keys(self, kstx_parent_transaction, b_return_packed_data):
        """
        Retrieve all keys from the database of items.

        """
        raise NotImplementedError, u'keys ' \
            + u'must be implemented in a subclass.'


    def reorganize_keys(self, kstx_parent_transaction):
        """
        Think of this like defragmenting a hard drive except that this function
        will consolidate gaps in IDs and free up "lost" space that was created
        with the passage of time. The reorganization is dependent upon the
        structure of column data store and requires that iteration through
        the list of keys is done in ASCENDING order [lowest to highest].
        Although it seems obvious, this function is only suitable for auto-
        generated integer or long IDs.

        All data must be throughly backed up using transactions. Because all
        changes are a part of a single transaction block, code from the
        function change_key_value_pair will be duplicated and send events
        will occur for every data change.

        """
        raise NotImplementedError, u'reorganize_keys ' \
            + u'must be implemented in a subclass.'


    def return_value_type(self, kstx_parent_transaction):
        """
        Return the data type of the stored value. This may be somewhat
        pointless if binary strings of data are stored. The pickling procedure
        for storing values should retrain the original object type.

        """
        raise NotImplementedError, u'return_value_type ' \
            + u'must be implemented in a subclass.'


    def generate_id(self):
        """
        This will generate an ID based on the time. Only eight new IDs can
        be generated per second. This means that 32-bit auto-generated IDs
        will run out of address space in about 4.5 years (30 new IDs per second
        for this length of time). The system constant for the maximum negative
        integer will represent NULL in the address space. Why 30...there
        are 30 images compose one second of video, so each frame would
        have its own ID. Therefore, all IDs will implicitly provide their
        date of creation.

        For this reason, (-sys.maxint-1) will be NULL and is not permitted
        as an ID unless the address space represents a virtual type that maps
        to a "universal primitive type" like an IEEE integer, float, Unicode
        character, etc.

        """
        mx_time_now = self.__now()
        if (self.__mx_last_datetime > mx_time_now):
            time_now = int(mx_time_now.gmticks() -
                            self.__mx_start_datetime.gmticks())

            if (self.__time_of_last_id == time_now):
                self.__i_ids_this_second = self.__i_ids_this_second + 1
                #print 'self.__i_ids_this_second = ', self.__i_ids_this_second

                if (self.__i_ids_this_second > KSUtility.MAX_IDS_PER_SECOND):
                    #print 'too many IDs...wait'
                    self.__i_ids_this_second = 0
                    while (self.__time_of_last_id == time_now):
                        time_now = int(self.__now().gmticks()
                                       - self.__mx_start_datetime.gmticks())
                        time.sleep(0.04)
                    # end loop
                # end if
            else:
                self.__i_ids_this_second = 0
            # end if

            #print 'now:', time_now
            #print 'last:', self.__time_of_last_id

            self.__time_of_last_id = time_now
            i_new_id = -sys.maxint + (time_now
                * KSUtility.MAX_IDS_PER_SECOND ) + self.__i_ids_this_second

            #print 'new_id:', i_new_id
            return i_new_id
        else:
            KSColumnError.print_and_raise_error('Safety feature: No new IDs '
                + u'can be automatically issued for this column. Only three '
                + u'(3) hours of automatic IDs remain. Reorganize all column '
                + u'IDs very soon.', KSColumnError.B_CLASS_EXCEPTION)


    def pack_key(self, obj_key):
        """
        Pack keys that are unicode or number types and store them in
        big endian notation to improve BTree ordering.

        """
        u_key = ''

        if (isinstance(obj_key, int)):
            u_key = struct.pack('>i', obj_key)
        elif (isinstance(obj_key, long)):
            u_key = struct.pack('>l', obj_key)
        elif (isinstance(obj_key, unicode)):
            u_key = obj_key  # default bsddb string encoding should be ok
        else:
            KSColumnError.print_and_raise_error('Key must be an integer, '
                + u'long integer, or unicode.',
                KSColumnError.B_CLASS_EXCEPTION)

        return u_key


    def unpack_key(self, obj_key, u_type):
        """
        Unpack keys that are unicode or number types for standard use.

        """
        u_key = ''

        if (u_type == 'int'):
            u_key = struct.unpack('>i', obj_key)
        elif (u_type == 'long'):
            u_key = struct.unpack('>l', obj_key)
        elif (u_type == 'unicode'):
            u_key = obj_key  # default bsddb string encoding should be ok
        else:
            KSColumnError.print_and_raise_error('Key must be an integer, '
                + u'long integer, or unicode.',
                KSColumnError.B_CLASS_EXCEPTION)

        return u_key[0]


    def pickle_data(self, obj_unpickled_data):
        """
        BerkeleyDB can only store strings of data, so store the data in a
        portable, byte string.

        """
        try:
            sio_buffer = StringIO.StringIO()
            pickle.dump(obj_unpickled_data, sio_buffer, -1)
            u_buffer = sio_buffer.getvalue()
            return u_buffer
        except Exception:
            raise

    def unpickle_data(self, obj_pickled_data):
        """
        Retrieve the original data from the pickled byte stream.

        """
        try:
            if (obj_pickled_data != None):
                sio_buffer = StringIO.StringIO(obj_pickled_data)
                obj_data = pickle.load(sio_buffer)
            else:
                obj_data = obj_pickled_data
            return obj_data
        except Exception:
            raise


    def custom_unpickle_data(self, obj_key, u_type):
        """
        Perform an unpickling of a key (usually is unpacked or packed).

        """
        # Ignore the u_type in this case in order to keep code from getting
        # to tangled.
        return self.unpickle_data(obj_key)


    def get_comparison_function(self, i_comparison_operator):
        """
        Retrieve a comparison function based on the symbol unicode operator.

        """
        try:
            fn_comparison = self.__function_comparison[i_comparison_operator]
        except KeyError:
            KSColumnError.print_and_raise_error('Comparison operator is  '
                + u'not valid.', KSColumnError.B_CLASS_EXCEPTION)

        return fn_comparison


    def change_datetimes(self, mx_start_datetime, mx_last_datetime):
        """
        Change the starting and ending datetimes for this column.

        """
        try:
            if (isinstance(mx_start_datetime, self.__type_mx_datetime)):
                self.__mx_start_datetime = mx_start_datetime
            else:
                KSColumnError.print_and_raise_error(u'A starting date '
                    + u'must be of type mx.DateTime.',
                    KSColumnError.B_CLASS_EXCEPTION)

            if (isinstance(mx_last_datetime, self.__type_mx_datetime)):
                self.__mx_last_datetime = mx_last_datetime
            else:
                KSColumnError.print_and_raise_error(u'An ending date '
                    + u'must be of type mx.DateTime.',
                    KSColumnError.B_CLASS_EXCEPTION)

        except Exception:
            KSColumnError.print_and_raise_error(u'An unknown error '
                + u'occurred while trying to change the column starting '
                + u'and ending dates.', KSColumnError.B_UNKNOWN_EXCEPTION)


    @staticmethod
    def calculate_current_epoch_time():
        """
        This function is used to calculate dates within Kaleidoscope. Dates
        from other time periods will have to perform their own calculations
        because the start time for this epoch is June 28, 2008. Below is the
        equation for this calculation:

        -sys.maxint+int((date.now()-KSUtility.DATE_EPOCH).seconds)

        """
        return int(mx.DateTime.now().gmticks()) - sys.maxint  \
            - KSUtility.DATE_EPOCH_SECONDS


    @staticmethod
    def convert_epoch_time_to_clock_time(i_epoch_seconds):
        """
        Conversion function that changes seconds since the system was created
        to a physical clock time.

        """
        date_clock_time = KSUtility.DATE_EPOCH + \
            mx.DateTime.DateTimeDeltaFromSeconds(
                i_epoch_seconds + sys.maxint)
        return date_clock_time


    @staticmethod
    def convert_string_key_type_to_int(u_key_type):
        """
        Convience function that returns the number associated with the
        key string type.

        """
        if (u_key_type == KSColumn.U_KEY_TYPE_INT):
            return 1
        elif (u_key_type == KSColumn.U_KEY_TYPE_UNICODE):
            return 2
        elif (u_key_type == KSColumn.U_KEY_TYPE_LONG):
            return 3
        else:
            u_message = u'KSColumnError.py: Key type name is not defined.'
            sys.stderr.write(u_message + u'\n')
            raise KSColumnError, u_message

    @staticmethod
    def convert_int_key_type_to_string(i_key_type):
        """
        Convience function that returns the number associated with the
        text name of a key type.

        """
        if (i_key_type == 1):
            return KSColumn.U_KEY_TYPE_INT
        elif (i_key_type == 2):
            return KSColumn.U_KEY_TYPE_UNICODE
        elif (i_key_type == 3):
            return KSColumn.U_KEY_TYPE_LONG
        else:
            u_message = u'KSColumnError.py: Key type number is not defined.'
            sys.stderr.write(u_message + u'\n')
            raise KSColumnError, u_message